{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.318722Z",
     "start_time": "2025-01-17T14:34:40.312404Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cpu\n",
      "cpu\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.383987Z",
     "start_time": "2025-01-17T14:34:40.372109Z"
    }
   },
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "print(housing.DESCR)\n",
    "print(housing.data.shape)\n",
    "print(housing.target.shape)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _california_housing_dataset:\n",
      "\n",
      "California Housing dataset\n",
      "--------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      ":Number of Instances: 20640\n",
      "\n",
      ":Number of Attributes: 8 numeric, predictive attributes and the target\n",
      "\n",
      ":Attribute Information:\n",
      "    - MedInc        median income in block group\n",
      "    - HouseAge      median house age in block group\n",
      "    - AveRooms      average number of rooms per household\n",
      "    - AveBedrms     average number of bedrooms per household\n",
      "    - Population    block group population\n",
      "    - AveOccup      average number of household members\n",
      "    - Latitude      block group latitude\n",
      "    - Longitude     block group longitude\n",
      "\n",
      ":Missing Attribute Values: None\n",
      "\n",
      "This dataset was obtained from the StatLib repository.\n",
      "https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html\n",
      "\n",
      "The target variable is the median house value for California districts,\n",
      "expressed in hundreds of thousands of dollars ($100,000).\n",
      "\n",
      "This dataset was derived from the 1990 U.S. census, using one row per census\n",
      "block group. A block group is the smallest geographical unit for which the U.S.\n",
      "Census Bureau publishes sample data (a block group typically has a population\n",
      "of 600 to 3,000 people).\n",
      "\n",
      "A household is a group of people residing within a home. Since the average\n",
      "number of rooms and bedrooms in this dataset are provided per household, these\n",
      "columns may take surprisingly large values for block groups with few households\n",
      "and many empty houses, such as vacation resorts.\n",
      "\n",
      "It can be downloaded/loaded using the\n",
      ":func:`sklearn.datasets.fetch_california_housing` function.\n",
      "\n",
      ".. rubric:: References\n",
      "\n",
      "- Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\n",
      "  Statistics and Probability Letters, 33 (1997) 291-297\n",
      "\n",
      "(20640, 8)\n",
      "(20640,)\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.394615Z",
     "start_time": "2025-01-17T14:34:40.390118Z"
    }
   },
   "source": [
    "# print(housing.data[0:5])\n",
    "import pprint  #打印的格式比较 好看\n",
    "\n",
    "pprint.pprint(housing.data[0:5])\n",
    "print('-'*50)\n",
    "pprint.pprint(housing.target[0:5])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,\n",
      "         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,\n",
      "         3.78800000e+01, -1.22230000e+02],\n",
      "       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,\n",
      "         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,\n",
      "         3.78600000e+01, -1.22220000e+02],\n",
      "       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,\n",
      "         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,\n",
      "         3.78500000e+01, -1.22240000e+02],\n",
      "       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,\n",
      "         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,\n",
      "         3.78500000e+01, -1.22250000e+02],\n",
      "       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,\n",
      "         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,\n",
      "         3.78500000e+01, -1.22250000e+02]])\n",
      "--------------------------------------------------\n",
      "array([4.526, 3.585, 3.521, 3.413, 3.422])\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.407152Z",
     "start_time": "2025-01-17T14:34:40.395124Z"
    }
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#拆分训练集和测试集，random_state是随机种子,同样的随机数种子，是为了得到同样的随机值\n",
    "x_train_all, x_test, y_train_all, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state = 7)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(\n",
    "    x_train_all, y_train_all, random_state = 11)\n",
    "# 训练集\n",
    "print(x_train.shape, y_train.shape)\n",
    "# 验证集\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "# 测试集\n",
    "print(x_test.shape, y_test.shape)\n",
    "\n",
    "dataset_maps = {\n",
    "    \"train\": [x_train, y_train],\n",
    "    \"valid\": [x_valid, y_valid],\n",
    "    \"test\": [x_test, y_test],\n",
    "}\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (11610,)\n",
      "(3870, 8) (3870,)\n",
      "(5160, 8) (5160,)\n"
     ]
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.415469Z",
     "start_time": "2025-01-17T14:34:40.408396Z"
    }
   },
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(x_train)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler()"
      ],
      "text/html": [
       "<style>#sk-container-id-2 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: #000;\n",
       "  --sklearn-color-text-muted: #666;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-2 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-2 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: flex;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "  align-items: start;\n",
       "  justify-content: space-between;\n",
       "  gap: 0.5em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label .caption {\n",
       "  font-size: 0.6rem;\n",
       "  font-weight: lighter;\n",
       "  color: var(--sklearn-color-text-muted);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-2 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-2 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-2 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 0.5em;\n",
       "  text-align: center;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-2 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>StandardScaler()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow\"><div><div>StandardScaler</div></div><div><a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.6/modules/generated/sklearn.preprocessing.StandardScaler.html\">?<span>Documentation for StandardScaler</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></div></label><div class=\"sk-toggleable__content fitted\"><pre>StandardScaler()</pre></div> </div></div></div></div>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构建数据集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.441438Z",
     "start_time": "2025-01-17T14:34:40.433462Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class HousingDataset(Dataset):\n",
    "    def __init__(self, mode='train'):\n",
    "        self.x, self.y = dataset_maps[mode]\n",
    "        self.x = torch.from_numpy(scaler.transform(self.x)).float()\n",
    "        self.y = torch.from_numpy(self.y).float().reshape(-1, 1)\n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.x[idx], self.y[idx]\n",
    "    \n",
    "    \n",
    "train_ds = HousingDataset(\"train\")\n",
    "valid_ds = HousingDataset(\"valid\")\n",
    "test_ds = HousingDataset(\"test\")"
   ],
   "outputs": [],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.461557Z",
     "start_time": "2025-01-17T14:34:40.452945Z"
    }
   },
   "source": [
    "train_ds[1]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.2981,  0.3523, -0.1092, -0.2506, -0.0341, -0.0060,  1.0806, -1.0611]),\n",
       " tensor([1.5140]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 18
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.471084Z",
     "start_time": "2025-01-17T14:34:40.466551Z"
    }
   },
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "batch_size = 8\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=False)\n",
    "val_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False)"
   ],
   "outputs": [],
   "execution_count": 19
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型\n",
    "\n",
    "多输出模型常见于多任务学习（Multi-Task Learning）中，一个任务会有一个输出层，计算一个loss，最后多个任务的loss联合训练模型。除此之外，如果想拿到模型的中间输出，也可以使用多输出。\n",
    "\n",
    "例如，这里想拿到 deep_output，我们构建多输出模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.478001Z",
     "start_time": "2025-01-17T14:34:40.473089Z"
    }
   },
   "source": [
    "#回归模型我们只需要1个数\n",
    "\n",
    "class WideDeep(nn.Module):\n",
    "    def __init__(self, input_dim=8):\n",
    "        super().__init__()\n",
    "        self.deep = nn.Sequential(\n",
    "            nn.Linear(input_dim, 30),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(30, 30),\n",
    "            nn.ReLU()\n",
    "            )\n",
    "        # pytorch 需要自行计算输出输出维度\n",
    "        self.output_layer = nn.Linear(30 + input_dim, 1)\n",
    "        \n",
    "        # 初始化权重\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"使用 xavier 均匀分布来初始化全连接层的权重 W\"\"\"\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Linear):\n",
    "                nn.init.xavier_uniform_(m.weight)\n",
    "                nn.init.zeros_(m.bias)\n",
    "        \n",
    "    def forward(self, x, return_deep_output=False):\n",
    "        # x.shape [batch size, 8]\n",
    "        deep_output = self.deep(x)\n",
    "        # concat [batch size, 30] with [batch size 8]\n",
    "        concat = torch.cat([x, deep_output], dim=-1)\n",
    "        logits = self.output_layer(concat)\n",
    "        # logits.shape [batch size, 1]\n",
    "        return (logits, deep_output) if return_deep_output else logits #如果return_deep_output为True，返回logits和deep_output，否则只返回logits"
   ],
   "outputs": [],
   "execution_count": 20
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.508662Z",
     "start_time": "2025-01-17T14:34:40.503535Z"
    }
   },
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = -1\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "outputs": [],
   "execution_count": 21
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:34:40.520273Z",
     "start_time": "2025-01-17T14:34:40.515653Z"
    }
   },
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for datas, labels in dataloader:\n",
    "        datas = datas.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # 前向计算\n",
    "        logits = model(datas)\n",
    "        loss = loss_fct(logits, labels)         # 验证集损失\n",
    "        loss_list.append(loss.item())\n",
    "        \n",
    "    return np.mean(loss_list)\n"
   ],
   "outputs": [],
   "execution_count": 22
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 训练(不重要，不用花时间看）"
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:03.663953Z",
     "start_time": "2025-01-17T14:34:40.535178Z"
    }
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, deep_output = model(datas,return_deep_output=True)\n",
    "                deep_output= deep_output.mean(axis=1).reshape(-1, 1) # # 平均池化,尺寸为[batch size, 30]，求平均就变为[batch size],reshape成[batch size, 1]\n",
    "                logits=logits+deep_output  # 尺寸一致，相加，求损失\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits, labels)\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "\n",
    "                    # 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 10\n",
    "\n",
    "model = WideDeep()\n",
    "\n",
    "# 1. 定义损失函数 采用MSE损失\n",
    "loss_fct = nn.MSELoss()\n",
    "# 2. 定义优化器 采用SGD\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.0)\n",
    "\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=10, min_delta=1e-3)\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=len(train_loader)\n",
    "    )"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/14520 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a63bfe8cd0a94d01bfd79c3c0acc1b7d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:03.792506Z",
     "start_time": "2025-01-17T14:35:03.664958Z"
    }
   },
   "source": [
    "#画线要注意的是损失是不一定在零到1之间的\n",
    "def plot_learning_curves(record_dict, sample_step=500):\n",
    "    # build DataFrame\n",
    "    train_df = pd.DataFrame(record_dict[\"train\"]).set_index(\"step\").iloc[::sample_step]\n",
    "    val_df = pd.DataFrame(record_dict[\"val\"]).set_index(\"step\")\n",
    "\n",
    "    # plot\n",
    "    for idx, item in enumerate(train_df.columns):\n",
    "        plt.plot(train_df.index, train_df[item], label=f\"train_{item}\")\n",
    "        plt.plot(val_df.index, val_df[item], label=f\"val_{item}\")\n",
    "        plt.grid()\n",
    "        plt.legend()\n",
    "        # plt.xticks(range(0, train_df.index[-1], 10*sample_step), range(0, train_df.index[-1], 10*sample_step))\n",
    "        plt.xlabel(\"step\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "plot_learning_curves(record, sample_step=500)  #横坐标是 steps"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGwCAYAAAAJ/wd3AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVXdJREFUeJzt3QeYU2X6NvA7mT7DzMAMbYCh996RIqI0sYFdQQV7/aPiZ2HXArpi23VRRF3dVWxYVgFZxQJIld57771PL5nJ+a7nTU7IzGQaJCfl3D+vYyrJeZNMzp23WjRN00BERERkEKtRT0REREQkGD6IiIjIUAwfREREZCiGDyIiIjIUwwcREREZiuGDiIiIDMXwQURERIYKR4Cx2+04cuQI4uPjYbFY/L07REREVAEybVhGRgbq1KkDq9UaXOFDgkdqaqq/d4OIiIguwMGDB1GvXr3gCh9S46HvfEJCglcf22az4ffff8egQYMQEREBs2C5WW6zMGvZWW6WOxCkp6erygP9OB5U4UNvapHg4YvwERsbqx43kN4wX2O5WW6zMGvZWW6WO5BUpMsEO5wSERGRoRg+iIiIyFAMH0RERGSogOvzQUREoamwsFD1V/AWeazw8HDk5uaqxzYLmx/LHRkZWe4w2opg+CAiIp/P/3Ds2DGcO3fO649bu3ZtNTrSTPNCaX4stwSPRo0aqRByMRg+iIjIp/TgUbNmTTVKw1sHTJmUMjMzE1WqVPHKr/FgYfdTufVJQI8ePYr69etf1PvI8EFERD4jzQJ68EhOTvb6wTA/Px/R0dGmCx/5fip3jRo1VAApKCi4qGG+5nm3iIjIcHofD6nxoOAX6Wxuudi+JgwfRETkc2bqkxHKLF56Hxk+iIiIyFAMH0RERGQohg8iIiIfa9iwISZOnOiVx5o/fz6qVavm9aHLRjLNaJeCQjuOp+fiVK6/94SIiIJBv3790LFjR6+EhpUrVyIuLs4r+xUKTFPzsetkJvq8tRD/3Bjm710hIqIQmexLhpxWdIgqR/yYMHzERToqefLs/t4TIiJzk4N2dn6BV7ac/MJK3V+euyJGjRqFBQsW4J133lEjPGSbMmWKOv3ll1/QpUsXREVFYfHixdi9ezeGDh2KWrVqqYm/unXrhjlz5pTZ7GKxWPDvf/8b119/vQolzZo1w8yZMy/4Nf3hhx/Qpk0btU/yXP/4xz+K3P7++++r55C5QWQ/b7rpJtdt33//Pdq1a4eYmBg1F8uAAQOQlZUFXzJNs0tclKOoNrtFNcFcxNwoRER0EXJshWj94m9+ee4tLw9GrPPHaFkkdOzYsQNt27bFyy+/rK7bvHmzOn3uuefw97//HY0bN1Z9L2Sa86uuugqvvvqqOvh//vnnuPbaa7F9+3Y1E2hpxo8fjzfffBNvvfUWJk2ahBEjRmD//v1ISkqqVJlWr16NW265BePGjcOtt96KJUuW4JFHHlFBQkLUqlWrMHr0aHzxxRfo1asXzpw5g0WLFql/K7OV3n777Wo/JAhlZGSo2yoa0i6UicJHWJEPfky0X3eHiIgCWGJioppQS2olZB0VsW3bNnUqYWTgwIGu+0pY6NChg+vyK6+8gunTp6uajMcee6zU5xg1apQ68IsJEybg3XffxYoVK3DllVdWal/ffvtt9O/fHy+88IK63Lx5c2zZskWFGnmOAwcOqP4m11xzDeLj49GgQQN06tTJFT6k6eiGG25Q1wupBfE104SPyDArwq0WFNg1ZOYVIine33tERGROMRFhqgbCG9OMZ6RnID4hvsLTjMtzX6yuXbsWuSzrrEitw88//+w6mOfk5KiDflnat2/vOi/hICEhASdOnKj0/mzdulU1+7jr3bu3auaRmUglKEmwkJoaCTay6c09EpokuEjgGDx4MAYNGqSaZKRGx5dM0+dD2tdiIx0fuux88yy9TEQUmN/H4V7ZYiLDKnV/b8zQWXzUyv/7f/9P1XRI7YU0Waxbt04dzGX9lbJEFGv/l32TQOVtUtuxZs0afP3110hJScGLL76oQocM1Q0LC8Ps2bNVP5bWrVur5p8WLVpg79698CXThA/3fh/S6YiIiKgs0uxSkTVM/vzzT9W8IbUJEjqkmWbfvn0wSqtWrdQ+FN8naX6RcCHCw8NVR1Lp27Fhwwa1f3/88Ycr9EhNifRBWbt2rSq3hClfMk2zi9BrPrLyWPNBRERlk1Ejy5cvVwdqGcVSWq2EjCKZNm2a6mQqB3Lpe+GLGozSPPXUU2qEjfQ1kQ6nS5cuxXvvvadGuIiffvoJe/bsQd++fVVzyqxZs9T+SQ2HlG/u3LmquUVWHpbLJ0+eVIHGl0xW8+EMH6z5ICKickhzitQcSHOEzNNRWh8O6fApB3UZSSIBRPpOdO7c2bD97Ny5M7777jt88803anSONKtIp1ipjRFVq1ZV4eiKK65QoeLDDz9UTTAyNFf6mSxcuFCN1pGakueff14N0x0yZIhP99lUNR9xzuFVrPkgIqLyyMFYahHc6Qf04jUkehOG7tFHHy1yuXgzjOZhKGtFp0uXmVfPnj2rgoPuxhtvVJsnffr0UVOyeyJh5Ndff4XRTFXzwQ6nRERE/meq8BGn13yw2YWIiALUQw89pPqYeNrktlBgqmaXWGefj2w2uxARUYB6+eWXVX8TT9ybWoKZqcJHnD7ahTUfREQUoGrWrKm20hg5ksZXTNrswpoPIiKioAkfMiRHhhLVqVNHjWeeMWNGqfeVtim5j/tKfoEw1JbNLkREREEUPmSZXZmWdfLkyWXeT2ZHW7ZsmQopgUJfyZDNLkREREHU50MmHilv8pHDhw/j//7v//Dbb7/h6quvRqDgUFsiIqIQ7HAqHWHuvPNOPP3002r2tPLk5eWpTZeenq5ObTab2rwp2rmYYWau9x87kOllNVOZBcttrnKbueyBXG7ZJ5lQS44N3u4oqU/UpT++WWh+LLc8nzyvvK/6ujG6ynz+vB4+3njjDbWAzejRoyt0/9dee00tZlPc77//rpb79aadabKaYRhOnE1Xc9ubjaxcaEYst/mYteyBWG45HshCa7LsfHmrvF6ojIwMBKL27dvj4YcfVlt5qlWrhi+//LJSrQX+KLe8hzk5Oar/Z0FB0S4M2dnZ/gkfq1evxjvvvKOW7q3ossVjx47FmDFjitR8pKamqkVuvD2eee3+03hvy2pYIqJx1VWXefWxA5mkUflSGjhwYIklnEMZy22ucpu57IFc7tzcXBw8eFBNkBUdHe3Vx5Zf4HIAliXjK3rMMZLValVlruixLCYmpkL39We55f2U/ZRF6oq/n3rLheHhY9GiRThx4gTq16/vuk6WI5YV92TEi6clhqOiotRWnPwBefuPKCE22tXnI9D+QI3gi9c0GLDc5mPWsgdiueUYIAdIORDL5k16k4P++IGoMvtmreBr5M9yy/PJ83r6rFXms+fVvZa+Hhs2bMC6detcm4x2kf4f0vk0kDqcelrUh4iIDCDfv/lZ3tls2ZW7fwW/+z/66CN1/Crep2Lo0KG45557sHv3bnW+Vq1aqlZHlrSfM2eO116ijRs3qlVopZYhOTkZDzzwgGq60i1evBiXXHIJ4uLi1Kq1vXv3xv79+9Vt69evx+WXX65qRqQmpUuXLli1ahUCSaVrPqTwu3btcl3eu3evChlJSUmqxkNepOJJSNr7WrRoAX+r4pzno8CuIa/AjuiIop1liIjIABIYJlz8NAzy67lqZf/RX44AkXHl3u3mm29WozbnzZuH/v37q+vOnDmjVoCVPoNyLJRl6F999VVVe//555+rObC2b99epPb/QmRlZWHw4MHo2bMnVq5cqVoU7rvvPjz22GOYMmWK6msxYsQI3H///fj6669VP4wVK1a4mmDktk6dOuGDDz5QnULlGB1oNWKVDh+SniRR6fT+GiNHjlQvSiCLcQsbUvvB8EFERKV1AJVpJaZOneoKH99//z2qV6+ujoHS/CBzXuleeeUVNb/VzJkzVUi4GFOnTlV9KyTQSM2GeO+991S4kUEdEiikf4V0Tm3SpIm6vVWrVq5/f+DAAdXi0LJlS3W5WbNmCDSVDh/9+vWrVJOFp34e/hIeZkWEVYPNbkFWXgGS4iL9vUtEROYTEeuogbhI0iSSnpGBhPj4ivd9kOeuIL124f3331e1G1999RVuu+029VxS8zFu3Dj8/PPPOHr0qKqNkFEgcuC/WFu3blXBRg8eQppVpLxSs9KnTx8MHz5chSPpZDxgwADccsstSElJcVUKSE3JF198oW6TWhw9pASKwOyh40NRzhJzllMiIj+R5gFp+vDGJmGiMvevxOgQqWmQH9sSMGTEjgyqkEAiZNVZqemYMGGCul6aNtq1a+ez4cTFySzjf/75J3r16oVvv/0WzZs3V7OKCwlFmzdvVjUjf/zxB1q3bq32NZCYL3w4W1qyuL4LERGVQYaS3nDDDarGQ/pWSN/Fzp07q9vkwD9q1Chcf/31KnRI30Zv1fS3atVKdRqVvh86eT6pcXHvPyn9OmS6iiVLlqBt27aquUYnYeTJJ59Uc2ZJGT799FMEEhOHD9Z8EBFR2aSmQ2o+PvnkE1eth96PYtq0aarGQ4KCNIN4a7bRESNGqOAjfSk3bdqkOr1K51cZUSqja2Sgh0zOuXTpUjXCRQLGzp07VWiRph/pczJ//nx1m4QW6bTq3ickEHh9htNgCR/ZbHYhIqJyyHBXGc0pfS0kYOjefvttNeRWmj2kE+qzzz5bqUm2yiKze8v0FI8//rgawiuXb7zxRvWc+u0SNqQvx+nTp1Vfj0cffRQPPvig6nsi19111104fvy42jep+fA0k7g/mS98WKWzrHQ4ZbMLERGVTZo6jhwp2Tm2YcOGqj+FOwkA7irTDKMVG8ghTTnFH18ntR8yFbvM4VG8o21kZKRqIgp05m12Yc0HERGRX5g3fLDmg4iIDCAdVmUWVE9bmwqs/h6KzNfswg6nRERkoOuuuw49evTweFtEgM08ahTzhQ/O80FERAaSNVZkI1M3uzg69bDmg4jION4ahkr+5a1FWcPN2+GUfT6IiHxNRl/oI0Zq1KihLusLoHkj0MiMorIOitFLy/uT3U/lluBx8uRJ9f5dbHORacNHNms+iIh8Tg6OjRo1UuufeBqyerEHQ5lUS5ad91agCQaaH8stz1evXj21uN3FMG344GgXIiJjSG2HLDMvE2AVFnrvu9dms2HhwoXo27evqTpu2vxYbnm+iw0epgwf0exwSkRkOL2q3psHSzkISqCRqcjNFD7CQqDc5mkkc4pkh1MiIiK/Ml34YIdTIiIi/zJd+Ihmh1MiIiK/Ml34iHT1+SiE3e6d8cpERERUcaat+RA5Nja9EBERGc104SPCKr2uHefZ6ZSIiMh4pgsfEjxiIx3VH+x0SkREZDzThQ9RJdIxvQlrPoiIiIxnyvDhqvlg+CAiIjKcKcNHXJSj5iObzS5ERESGM3XNRyZrPoiIiAxnyvAR55zmNJvruxARERnOlOEj1tnhNJMr2xIRERnOlOEjztnswinWiYiIjGfqDqec54OIiMh4pgwfHGpLRETkP6bucJrFDqdERESGM3WHU9Z8EBERGc/cHU7Z54OIiMhwJg0frPkgIiLyF1OGj1i9zwfn+SAiIgr88LFw4UJce+21qFOnDiwWC2bMmOG6zWaz4dlnn0W7du0QFxen7nPXXXfhyJEjCMRmF3Y4JSIiCoLwkZWVhQ4dOmDy5MklbsvOzsaaNWvwwgsvqNNp06Zh+/btuO666xBI4tjsQkRE5DeOo3AlDBkyRG2eJCYmYvbs2UWue++999C9e3ccOHAA9evXL/Fv8vLy1KZLT0931aLI5k3640WGaa5Jxrz9HIFIL6MZyuqO5TZXuc1cdpab5Q4Eldkfi6ZpjiPxBZBml+nTp2PYsGGl3mfOnDkYNGgQzp07h4SEhBK3jxs3DuPHjy9x/dSpUxEbGwtfyLIBf1nlyF1v9yhAmCl7vhAREXmPtH4MHz4caWlpHo/3hoWP3Nxc9O7dGy1btsRXX33l8T6eaj5SU1Nx6tSpcnf+QlKZ1MxcdkV/dHx1gbpu1V8uR2JMBEKZXu6BAwciIiK0y+qO5TZXuc1cdpab5Q4EcvyuXr16hcJHpZtdKvPi3HLLLZBs88EHH5R6v6ioKLUVJy+or17UuOgoRIZZkV9oR77dElBvni/58jUNZCy3+Zi17Cy3uUQEWLkrsy9WXwaP/fv3q3Tm7RoM7w63ZadTIiIiI1l9FTx27typ+nskJycjEMXpI144yykREZGhKt3skpmZiV27drku7927F+vWrUNSUhJSUlJw0003qWG2P/30EwoLC3Hs2DF1P7k9MjISAbe4HGs+iIiIAjt8rFq1Cpdffrnr8pgxY9TpyJEj1ciVmTNnqssdO3Ys8u/mzZuHfv36IVBwcTkiIqIgCR8SIMoaIHMRg2cMVSXKUXQuLkdERGQs085wEeucYj2TNR9ERESGMm34OF/zwfBBRERkJNOGD32obSZXtiUiIjKUacNHnF7zwWYXIiIiQ5k3fLjm+WD4ICIiMpLV7B1Os9jsQkREZCjThg92OCUiIvIP04aPWGf44FBbIiIiY5k2fFRxjnbhJGNERETGMm340KdXZ80HERGRsUwbPuKc4SObHU6JiIgMZd7woa9qyw6nREREhjJx+Di/qm2wLIZHREQUCkwfPuwakFdg9/fuEBERmYZpw0dshKPZRbDTKRERkXFMGz6sVotrllN2OiUiIjKOacOH4HBbIiIi45k6fOgjXjjFOhERkXHMHT5cK9uy2YWIiMgo5g4f+lwfbHYhIiIyjMnDx/m5PoiIiMgY5g4ferMLwwcREZFhTB0+9KG27PNBRERkHFOHD73ZhaNdiIiIjGPy8KF3OGXNBxERkVFMHj7Y54OIiMho5g4frnk+GD6IiIiMYu7w4ar5YLMLERGRUcwdPvSF5VjzQUREZBhTh49YZ81HJms+iIiIDGPq8FGFC8sREREZztThI5YznBIRERnO1OGjCjucEhERGc7U4UOfXj3HVohCu+bv3SEiIjKFSoePhQsX4tprr0WdOnVgsVgwY8aMIrdrmoYXX3wRKSkpiImJwYABA7Bz504E8lBbwX4fREREARo+srKy0KFDB0yePNnj7W+++SbeffddfPjhh1i+fDni4uIwePBg5ObmItBEhVsRZrWo89lcXI6IiMgQ53/6V9CQIUPU5onUekycOBHPP/88hg4dqq77/PPPUatWLVVDcttttyGQSM2NNL1k5BYgM68Atfy9Q0RERCZQ6fBRlr179+LYsWOqqUWXmJiIHj16YOnSpR7DR15entp06enp6tRms6nNm/THc3/cOGf4SM/Kg61qFEKRp3KbActtrnKbuewsN8sdCCqzP14NHxI8hNR0uJPL+m3Fvfbaaxg/fnyJ63///XfExsbCF2bPnu06r9mk06kFcxf+iQOJod3p1L3cZsJym49Zy85ym8vsACt3dna2f8LHhRg7dizGjBlTpOYjNTUVgwYNQkJCgtdTmbxZAwcOREREhLruPweW4fjhdLTr3BVXtKiBUOSp3GbAcpur3GYuO8vNcgcCveXC8PBRu3ZtdXr8+HE12kUnlzt27Ojx30RFRamtOHlBffWiuj92XJTjVKb6CKQ30Rd8+ZoGMpbbfMxadpbbXCICrNyV2RevzvPRqFEjFUDmzp1bJAnJqJeePXsiEMU5p1jnLKdERETGqHTNR2ZmJnbt2lWkk+m6deuQlJSE+vXr44knnsDf/vY3NGvWTIWRF154Qc0JMmzYMATyXB8MH0RERAEaPlatWoXLL7/cdVnvrzFy5EhMmTIFzzzzjJoL5IEHHsC5c+fQp08f/Prrr4iOjkZgr+/CeT6IiIgCMnz069dPzedR1twZL7/8stqCAVe2JSIiMpap13Zxr/mQScaIiIjI90wfPvQOp5xenYiIyBgMH+xwSkREZCiGD73DKft8EBERGYLhw1XzwWYXIiIiIzB8RHKSMSIiIiMxfDhrPtjhlIiIyBgMH/r06uzzQUREZAjTh4/zM5wyfBARERnB9OFDb3axFWrIL7D7e3eIiIhCHsOHs8OpYO0HERGR75k+fISHWREV7ngZ2O+DiIjI90wfPgTn+iAiIjIOw4fqdMoRL0REREZh+ABQRZ/rgzUfREREPsfw4VbzkckOp0RERD7H8FFkllOGDyIiIl9j+HBf2ZY1H0RERD7H8OE+2oXruxAREfkcw4fb+i7ZrPkgIiLyOYYPt/VdMjnahYiIyOcYPtRQW2fNBzucEhER+RzDR5GaD4YPIiIiX2P4cJ9kjB1OiYiIfI7hQ2o+nM0uHGpLRETkewwf7vN8sM8HERGRzzF8uM9wytEuREREPsfwwbVdiIiIDMXwwQ6nREREhmL4cO9wml8ATdP8vTtEREQhjeHDreZDckeOjbUfREREvsTwASA6PAwWi+N8FjudEhER+RTDh7wIVgtiIzjXBxERkREYPooNt+VcH0RERL7F8FE8fLDZhYiIKLjCR2FhIV544QU0atQIMTExaNKkCV555ZWAH0US5zbihYiIiHzH8XPfi9544w188MEH+Oyzz9CmTRusWrUKd999NxITEzF69GgE+sq2nOWUiIgoyMLHkiVLMHToUFx99dXqcsOGDfH1119jxYoVHu+fl5enNl16ero6tdlsavMm/fE8PW5MhKMSKC07z+vP629llTuUsdzmKreZy85ys9yBoDL7Y9G83B4yYcIEfPTRR/j999/RvHlzrF+/HoMGDcLbb7+NESNGlLj/uHHjMH78+BLXT506FbGxsTDKlB1WrD1txQ0NC3FZSmA3EREREQWa7OxsDB8+HGlpaUhISDA2fNjtdvzlL3/Bm2++ibCwMNUH5NVXX8XYsWMrXPORmpqKU6dOlbvzF5LKZs+ejYEDByIiIqLIbX+ZsRn/XX0YT/Zvikf6NUYoKavcoYzlNle5zVx2lpvlDgRy/K5evXqFwofXm12+++47fPXVV6rmQvp8rFu3Dk888QTq1KmDkSNHlrh/VFSU2oqTF9RXL6qnx46PjlSnuYVaQL2Z3uTL1zSQsdzmY9ays9zmEhFg5a7Mvng9fDz99NN47rnncNttt6nL7dq1w/79+/Haa695DB8BN9qFk4wREREF11BbafOxWos+rDS/SHNMIOM8H0RERMbwes3Htddeq/p41K9fXzW7rF27VnU2veeeexDI4iIdNR/ZnOeDiIgouMLHpEmT1CRjjzzyCE6cOKH6ejz44IN48cUXEcj0eT4y2exCREQUXOEjPj4eEydOVFsw0ZtdsvPZ7EJERORLXNvFiR1OiYiIjMHw4cRVbYmIiIzB8OEUx7VdiIiIDMHw4RTrHO3CDqdERES+xfDhVMXZ7JJXYEdBYWDPSUJERBTMGD6cYp0dTkUWR7wQERH5DMOHU1R4GCLCLOo8JxojIiLyHYYPDxONcbgtERGR7zB8eOj3wfVdiIiIfIfhw8OIF871QURE5DsMH25iWfNBRETkcwwfbqo4R7ywwykREZHvMHy44cq2REREvsfw4aHDKadYJyIi8h2GDzfscEpEROR7DB+eVrZlswsREZHPMHy4idMnGeP06kRERD7D8OEmzjnahTUfREREvsPw4bHZhTUfREREvsLw4aHDKef5ICIi8h2GD49ruzB8EBER+QrDh6dVbdnhlIiIyGfMEz40DZYja1D73OpS78IOp0RERL5nnvCxay7CPx2EDgc/A+yeazY4zwcREZHvmSd8NLoUWlQCogvOwXJwmce7xLk1u2iaZuz+ERERmYR5wkd4FLQW16izli0zymx2KbRryCuwG7p7REREZmGe8AHA3nqYOrVu+x9QWFBqh1ORzU6nREREPmGq8KE1vBR5YVVgyT4F7FtY4vYwqwXREY6XhP0+iIiIfMNU4QNhEThStZvj/KZpZc/1wYnGiIiIfMJc4QPAkWo9HGe2zgQK8kuf64M1H0RERD5huvBxqkpLaHE1gdw0YM+8ErdzfRciIiLfMl34gMUKe6uhpTa9xHF9FyIiIp8yX/iQjqfOUS/Y9jNgy/VY85HJmg8iIiKfMGf4qNcNSKgH5GcAu2Z7nOuDNR9ERERBFD4OHz6MO+64A8nJyYiJiUG7du2watUqBAyLFWjjrP3Y9IPHDqeZ7HBKREQUHOHj7Nmz6N27NyIiIvDLL79gy5Yt+Mc//oFq1aohoLS90XG64zcgP6vEUNtsNrsQERH5xPkpPb3kjTfeQGpqKj799FPXdY0aNULAqdMJqNYQOLsP2PGrK4zEOjucsuaDiIgoSMLHzJkzMXjwYNx8881YsGAB6tati0ceeQT333+/x/vn5eWpTZeenq5ObTab2rxJfzz91NpqGMKWTIR9w/cobHGdui4m3KJOM3O9//z+UrzcZsFym6vcZi47y81yB4LK7I9F8/LyrdHR0ep0zJgxKoCsXLkSjz/+OD788EOMHDmyxP3HjRuH8ePHl7h+6tSpiI2N9eaulZCQcwCXb3sehZYI/NruPRSExWDhUQt+2BeGTsl2jGrOxeWIiIgqIjs7G8OHD0daWhoSEhKMDR+RkZHo2rUrlixZ4rpu9OjRKoQsXbq0QjUf0mxz6tSpcnf+QlLZ7NmzMXDgQNUnBZqG8H/1guX0ThRc9z60drfg+zWHMXb6ZlzWrDr+fVdnhIIS5TYJlttc5TZz2VluljsQyPG7evXqFQofXm92SUlJQevWrYtc16pVK/zwQ9FRJbqoqCi1FScvqK9e1CKPLX09FryO8K0zgM4jkBjr2Jccmz2g3lRv8OVrGshYbvMxa9lZbnOJCLByV2ZfvD7aRUa6bN++vch1O3bsQIMGDRCQ2t7gON39B5B9hh1OiYiIfMzr4ePJJ5/EsmXLMGHCBOzatUv13fjoo4/w6KOPIiDVaAHUagvYC4Ct/zs/1JaTjBEREQVH+OjWrRumT5+Or7/+Gm3btsUrr7yCiRMnYsSIEQhYeu3H5mnnV7XN5zwfREREvuD1Ph/immuuUVvQaHMDMPdlYO9CJBSeU1dlsdmFiIjIJ0y5tksJSY0ck45pdlTd97O6Kju/EHa7VwcCEREREcOHG+cMpzE7Zrquyrax6YWIiMjbGD50ba5XJ9aDS5FiOaPOZ7PphYiIyOsYPnSJ9YDUHrBAw7DIleoqDrclIiLyPoYPD00vV1mXuvp9EBERkXcxfLhrPVRmnEc7bQfqWU5yxAsREZEPMHy4i68NNOyjzl5tXYYsTjRGRETkdQwfpUw4dk3YUmTlsdmFiIjI2xg+ims1FIWwop11Hyxndvt7b4iIiEIOw0dxccnYEdtFnU05+Iu/94aIiCjkMHx4sDmpvzpteIzhg4iIyNsYPjzYV/1y5GthSM7eA5zY6u/dISIiCikMHx5Y45KwwN7BcWHTNH/vDhERUUhh+PAgLjIMPxVe4riw6QdA4wJzRERE3sLw4UFcVDjm2Lsg3xIJyIiXYxv8vUtEREQhg+HDg7ioMGQhBuuiu5+v/SAiIiKvYPjwIC4yXJ0uiLjUccXm6Wx6ISIi8hKGj1KaXcQidAYi4oBzB4DDq/29W0RERCGB4cOD2MgwdXo6PxxoMcRxJZteiIiIvILhw4MqzpqPbFlYru2Njis3zwDsdv/uGBERUQhg+PAg1hk+1MJyTfsDUYlAxhHg4DJ/7xoREVHQY/jwoIqzw2l+oR35iABaXu24gU0vREREF43hw4PYKEefD5GTX3i+6WXLj0Bhgf92jIiIKAQwfHgQEWZFZLjjpcmUfh+NLwNikoCsk8C+Rf7ePSIioqDG8FHGFOsiO68ACIsAWl3ruGEz13ohIiK6GAwfpYh19vvIlPAh9KaXrf8DCvL9uGdERETBjeGj3OG2hY4rGvYB4moCOWeBPfP9u3NERERBjOGjnE6nWXrNhzUMaDPMcZ5NL0RERBeM4aOcmo8s6XCqa3OD43TrT4At1097RkREFNwYPsqZYl1NNKZL7QHE1wHyM4Bdc/y3c0REREGM4aMUcc4Op65mF2G1Am2dtR9seiEiIrogDB/lrGybpXc4Ld70sv0XID/LD3tGREQU3Bg+KtrhVFe3M1C1AWDLBnb85p+dIyIiCmIMH+Ws76JWtnVnsbDphYiI6CIwfFRkZdvi9KaXHb8DuekG7xkREVFw83n4eP3112GxWPDEE08gmFQprdlF1G4HJDcDCvMcfT+IiIgoMMLHypUr8a9//Qvt27dHsE6vXmSeD09NL5t+MHjPiIiIgpvjCOsDmZmZGDFiBD7++GP87W9/K/V+eXl5atOlpzuaMWw2m9q8SX+8ijyus+IDmbkFnu/f4jpELHgD2u4/UJB+EoipikBVmXKHEpbbXOU2c9lZbpY7EFRmfyyapmm+2ImRI0ciKSkJ//znP9GvXz907NgREydOLHG/cePGYfz48SWunzp1KmJjY+Evu9KBSZvDUTNaw187eej3AaDf1r8iMfcg1ta/FweSLzN8H4mIiAJFdnY2hg8fjrS0NCQkJBhf8/HNN99gzZo1qtmlPGPHjsWYMWOK1HykpqZi0KBB5e78haSy2bNnY+DAgYiIiCjzvpuPpGPS5mWwREbjqqs8BwtrwjZgwQR0CNuNtle9gUBVmXKHEpbbXOU2c9lZbpY7EOgtFxXh9fBx8OBBPP744+qFiY6OLvf+UVFRaitOXlBfvagVeezEOMe+Z+cVln7f9jep8GHdtwjW/DQgrjoCmS9f00DGcpuPWcvOcptLRICVuzL74vUOp6tXr8aJEyfQuXNnhIeHq23BggV499131fnCQs9NGIEmTl/bJb8ApbZMJTcBUjoCWiGw5Udjd5CIiChIeT189O/fHxs3bsS6detcW9euXVXnUzkfFubsyRkk83zYNSDXZi/9jq5RL5xwjIiIyC/NLvHx8Wjbtm2R6+Li4pCcnFzi+kAWG3E+JEntR4yzJqSENtcDs18E9v8JpB8FElKM20kiIqIgxBlOS2G1WhAbWcZEY7qq9YF63QFowJYZxu0gERFRkPLZPB/u5s+fj2Bd2TY7v9DzFOvFm14OrXA0vVzysFG7R0REFJRY81GBTqclFpcrrvUwmTLFEUDOHTBm54iIiIIUw0c5NR8is6xmFyH9PBr0dpzfPN2APSMiIgpeDB9liHOu7yJNL+XiqBciIqIKYfgoQ6xzgZdyaz5E66GAJQw4ug44vdv3O0dERBSkGD4q0OySXZHwIbObNurrOL+ZtR9ERESlYfio0CynFZyVte2NjtNN7PdBRERUGoaPCtR8lDnPh7tW1wDWCODEZuDENt/uHBERUZBi+ChDXGU6nIqYakCTKxzn2fRCRETkEcOHtzqclmh6mQaUtiAdERGRiTF8lKGK3uG0vEnG3LUYAoRFAad3Asc2+m7niIiIghTDRxlinc0umeVNr+4uOgFoPshxnk0vREREJTB8lKGKs9mlQkNt3bXRJxz7gU0vRERExTB8VKDmo8JDbXXNBwMRsY51Xg6v8c3OERERBSmGD28OtdVFxjn6fgg2vRARERXB8FGGOL3ZpTIdTks0vUwD7HYv7xkREVHwYvgoQ5yrw+kFhI+mA4CoBCDjCHBwudf3jYiIKFgxfFSg2SXXZkehvZIdRyOigZZXn+94SkRERArDRxlinWu7XHTTy5YfAXslO60SERGFKIaPMkSFWxFutajzWZWZ60PXuJ9jyvWsE8C+xd7fQSIioiDE8FEGi8Xiqv3IupCaj/BIoNW1jvNseiEiIlIYPnw13LZ408vWmUChzYt7RkREFJwYPiocPi6wz0bDS4G4GkDOWWDPAu/uHBERURBi+ChHnLPZ5YI6nIqwcKD1UMd5Nr0QERExfFS05uOC5voo3vSy7SegIM9Le0ZERBScGD4quL5LdmXXd3FXvycQnwLkpQO75nhv54iIiIIQw0cFV7a94A6nwmoF2lx/frp1IiIiE2P4KEfsxXY41bW90XG6/RcgP9sLe0ZERBScGD4q2OH0gub5cFe3C1C1PmDLAnb+5p2dIyIiCkIMH76e50NnsbDphYiIiOGjfHHe6HBavOll5+9AXsbFPx4REVEQYvgwYqitrnZ7IKkJUJDr6PtBRERkQgwf5YiLushJxoo3vei1H2x6ISIik2L4qOA8H5kXO9pF19Y54ZjM9yFTrhMREZkMw0dFaz680ewiarYCarQC7DZg28/eeUwiIiIzh4/XXnsN3bp1Q3x8PGrWrIlhw4Zh+/btCFZx3uxwqmPTCxERmZjXw8eCBQvw6KOPYtmyZZg9ezZsNhsGDRqErKwswOwdTos3veyZD2Sd8t7jEhERBQHHkdWLfv311yKXp0yZompAVq9ejb59+8LUHU51yU0cI1+ObQC2zgS63uO9xyYiIjJb+CguLS1NnSYlJXm8PS8vT2269PR0dSo1JrJ5k/54lXncKKvm+DeFGjJz8hAV7p3KImvrYQg7tgH2jd+jsMOd8KULKXcoYLnNVW4zl53lZrkDQWX2x6JpmuPo6gN2ux3XXXcdzp07h8WLF3u8z7hx4zB+/PgS10+dOhWxsbHwt0INGLPMkdEmdC1AXIR3Hjcm7yQGbXkKGiz4re07yIuo6p0HJiIi8oPs7GwMHz5cVTokJCT4L3w8/PDD+OWXX1TwqFevXoVrPlJTU3Hq1Klyd/5CUpn0Qxk4cCAiIiqeItqOn4O8AjvmjbkU9arFeG1/wqZcCevhVSgc9Brs3e6Hr1xouYMdy22ucpu57Cw3yx0I5PhdvXr1CoUPnzW7PPbYY/jpp5+wcOHCUoOHiIqKUltx8oL66kWt7GNXiQpHXkE+8u0W7+6TjHo5vAphW2cgrNcj8DVfvqaBjOU2H7OWneU2l4gAK3dl9sXro12kIkWCx/Tp0/HHH3+gUaNGCHaxUV5a2ba4NsOk8gk4uBw4d9C7j01ERBSgvB4+ZJjtl19+qfpsyFwfx44dU1tOTg6CVVykl1a2LS6hDtCgl+P85unefWwiIiKzhI8PPvhAtff069cPKSkpru3bb79FsNLn+sjy1hTr7tpc7zjdzAnHiIjIHLze58OH/Vf9JjYyzDc1H6L1UOCXZ4Aja4Eze4Ckxt5/DiIiogDCtV0q2OHU6xONuR68JtDIOfkap1snIiITYPioxMq2Wd5c38VdG+d06+z3QUREJsDwUQFVonzY7CJaXQtYw4Hjm4CTwbsIHxERUUUwfFRArC87nKonSAKaXOE4z6YXIiIKcQwflejz4bOajyJNL9Ok167vnoeIiMjPGD4qM9rFFx1OdS2vAsKigFM7HM0vREREIYrhowLifDXJmLvoRKDZQMd5Nr0QEVEIY/iozCRjvhrt4mnCMTa9EBFRiGL4qMTaLj6Z58NdiyFARCxwdh9wZI1vn4uIiMhPGD4q1eHUxzUfkXFA88GO82x6ISKiEMXw4e/p1Ytre+P5Ccfsdt8/HxERkcEYPiogzogOp7qmA4HIeCD9MHBohe+fj4iIyGAMH5XocJptK4Td7uOOoBHRjmG3gk0vREQUghg+KiDO2eFUBqDkFvi434d708uWGYDdgOcjIiIyEMNHBcREhMFicZzPNKLppfHlQHRVIPM4sP9P3z9fiNp5IhNLj1uQZ2OAIyIKJAwfFWCxWFz9PrJ9PeJFhEc6FpsTbHq5IL9sPIobPlyGb/aE4eaPVmDvqSx/7xIRETkxfFSy6cWQmg/R1rnWy5YfgUKbMc8ZAjRNw+R5u/DwV2uQa7PDCg1bj2XgmncX4cd1h/29ewHJVmjH9mMZmLH2MF6btRV3fbICz36/AbmsMSIiH3H8nKdyxamajzxk+3qWU13DvkBsdSD7FLB3AdB0gDHPG8TyCgox9oeNmLbWETLuuqQ+GufvwS9na2D53rN4/Jt1WLr7NF66tg1inMOnzeZsVj62Hk3HlqPp2Ho0Q53fdSIT+YUlh3WfyszDB3d0QWQ4f6MQBZKFO0/hx31W9Mq2oUZiBIIRw0clZzn16eJy7sLCgdZDgVX/cTS9MHyUSQ6UD36xGqv3n0WY1YLx17XBrV3qYNasPfhsVFe8v3AfJv2xE9+sPIi1B85h8ohOaFozHqFKBmXtPpmFnSezVcBwbBk4lp5b6kR6rVLi0bJ2AlKqRuOdOTsxd9sJjP56Ld4b3gnhYQwgRIFg/vYTeODLtSi0W3HHJyvxxX09UDM+GsGG4aOC4oyc68O96UXCx/pvgEMrgYS6QGJdIKEekFjP7Xxdx+yoJiVNBvd+thKHzuYgPjocH4zogj7NqsNmczRXSRgZM7A5LmmUhMe/XYftxzNw7aQ/8cqwtripSz2EmqkrDmLCijDkLfPcWbl+UqwKGq1SEtTWOiUB9arFqL5NurZ1EnHfZ6vw6+ZjGPPdevzz1o7qdSQi/9lw6Bwe+WoNCu2aalLefjwTN3+4FF/e2wOpSbEIJgwflZ3rw4gOp7r6PYHa7YBjG4FTOxxbaWR0jAok9YqFlLpAbC1Y7AaGJgPN23YC//f1WtUXp0FyLP4zshua1qzi8b69mlbHrNGX4slv12HxrlP4f/9djyW7T+GVoW1d72+w+2jhbkyYtU26SSMmwooWtfWA4QgbLWrHIz66/Gravs1r4P0RnfHQl6sxc/0RRIVb8caN7WFlACHyi/2ns3DPlJWq6b93k2RcFn8cn++Px/7T2bjpwyUqgDSrFTy1uaHxjWsA/eB0MjPPuCe1hgH3z3csNJd+CEg77Jj5NO2Q89R5Pj8DyD3n2I5vKvEwcqi5Tjpj7qpZssZEBZVUx/kqtRzPGSQdSz/9cx/+9vMW1cTQo1ESPryjC6rFRZb572rER+Hze7rj/fm78PbsHZi25jDWHzyH94Z3VgfnYDZp7k78Y7YjoA6sa8ekBwYiOqrs16MsA1rXwqTbO+Gxr9fiv6sPISrCqoKaew0JERnTrHzXJytwKjMfbeok4L3bO2Dh3N/xzX3dcM/na7DjeCZu+ddSfHZPd7SvVxXBgOGjgqrGOH4tvvXbdvy+5Thu7lIP13aog0Tn9T4TFo6D1jrYklMFPVr0QtVYDweT3LRSgslBdV5LOwxLYR4sWScA2UpbMdcaDsSnuNWcuAUTdb4eEJssY4/h79EZL83cjKnLD6jLt3ZNVU0oFe0YKb/eH7uiGbo1TMLob9aqvhHDJv+pOqLe3j016A6uEsT+/vt2TJ63W11+on9TNMre5pVmkiHtUvB2oR1PfLsOXy47gMiwMLxwTauge40q2hlXflw0D6JfjxT6svIKcO+UlaqGIzUpBp/e3Q1VnH0QayVE49sHemLUlJXqR9Twj5fj47u6omeTZAQ6ho8Kuv/SxqqznlTzy5ss28s/bcHgNrVVEOndtLrX2sRlCveNh9Mwe8txtUkfBVE1NgJPD26B27rVL/pc0YmOrVZrj49XkJ+POTO/xYDurRCRfbxIMHGFlvQjgDTNyPWyHSxl58KjgYQ6noOJHlpkX3wkLduGR6auxp+7TqsM9JchrXDfpY0u6GDYo3GyaoZ56r/rMX/7Sfxl+kYs3XMaE65vW6GmiUAJHn/7eSv+s3ivuvzXq1phVM9UzFJNL94xtGNd5NnseOaHDfjkz72qBuSZwS1CJoDIr8qPF+7B50v3I8dWiLt7N8TYIa04yof8zlZox6NT12D9oTRUi43AZ3d3V51L9f5sQmp7v7qvBx74fBWW7D6NkZ+uwPvDO6uay0DG8FFB9ZNjVaI8mZGn5ov476pDKhT8b/0RtaUkRuPGzvVUB8aG1eMuaJioDAOVsDFn63EcTz/fvCNBo3qVSHXdX6dvwjcrDmL80DboXL9axR7cYkF+RAKQ0gGIKOWgKtO4Z3oKJm7NPXJ7QS5wZo9jK01UAhBXA4iIAcKjHIElLNJxql8OL345CgiLKvM+hzM1vPTTThw5V4AmkdF4YVhn9GudDBTmOx7/Ag6GyVWi8MnIbvh40R68+dt29V5uPORohmlb13chylsh9cWZm1SNhHh5aBvc1bNhkS8mb7mlWyryCu14YcYmfDB/N6LDw/D4gGYIZqcz8/DRoj34fIkjdOikOW/DoTRMHt4ZtRODbxSB0Q6eyVa1iXWrxvh7V0KKpmn4y7SN6odRdIQVn4zqhsY1qpQ6Wk1ul/5vcgx58MvV+MfNHTCsU10EKoaPSpI+A/dd2hj39mmkaickhEgYOZqWi/fm7VJbt4bVcHOXVFzVPkV9KMr6Ff/HdkftxoLtJ5HlNodIbGQYLmteAwNb18IVLWuqx/li2X68/fsO9bw3vL8Et3Sth2eubInqVaIuvmDS10PVaNQBUrt5vk9BnqOGxFMw0S/nnAXy0h2bl8mf0b/ljF7cmc5NVyzEhIdHol92PsKOv+NYsK9E0HFctoZH4cHwaFxziR0zNxzDmXN2zPzwB2S1qYPujWvAIs1R8vrIqUVOw4pdltutxS4771PkstzuvF+5jxdWZpiS3u7P/bBB9cWQu71+Qzvc2q0+fOnOSxqoqeqlpuWfc3aoGpCHLmuCUAgd7esl4okBzVBQqKmaMBmyfc2kRXj3tk6qozKVJLW/Mnx9ztYT6jM4qHUtPNyvKTqmBkefg0D39uwd6u9bfnxKEO5Uzo/N6IgwfDCis6qhlL5sT363Dhm5NtzZsyECkUWTeBVA0tPTkZiYiLS0NCQkeLcDoPwinDVrFq666ipElFYDcAFkJkiprZAgsmjnSdUBUg8QQ9qm4Oau9dC9YZL6dSC/EvTmlBX7zqiDiK5mfJSqKpPA0bNxsvowFSc1L2/8ug3frz6kLidEh+OpQS0wokf9UudiuJhySw9rmdBG+rw0qVEFjWvEedwvl/wsRxCRydEkrKgt13Fa6H7ZeV3x+8ip1GS4XT6bnokz6RmIQj7irIVIjCyEVb9PKJOg4iHMaNZwpOXZkWkD7LAgqUoMqkRLzY/jPhosSMvIRELVarDqAUiFHuep63xp1+vP6/n69UcysGp/GgphRc8m1dEuNanYYzmDU7nP4Xwe13mr279zv879fpZSHsexFRQWYvGSZejd51JEREQWue/ZnAJ8u+owflh7BNn5drX/LVMScV/fJujdrCYsqsxW7D+Tiye+W49tx7PUv318YEs80Lep87WU5w+85iZffbd5IsFMQof8IhfycrgfRXo1ScbD/ZqgT9PqPm+aM7LcRvpy2X48P8MxeEB+WNzWvX6Fyy01otIlYMqSfeqyNNU/0q+JIc2klTl+M3x42bG0XPyw5hB+WH0Ie9zWE5G5FSSMbDvm6L+ha16rigobA1vXRvu6iRUeyrh6/xm8+ONmbD7iqGFoWTseLw9ti+6Nki663IfP5eDnDUfw04ajqvrZnXx+U6vFokmNODWkVQKJnMrmsTNsJWXnF+BUhqPjnwStxbtOupoVrutQB2/e1P58+JGPbpGgUjTIFORlYsWSxejeuT3CUVBG2HG7zm6HphVgz/E0bD1yDlatEGGwo3aVcDSuHoN4efm0QkczldoKnJcL1L8t+7Lz37hflvMUfPRQA7cwVCQcWcq53Xm+zNv1f28p53YrZH7a06fPILlGDWdIKr5/+v6glP3zdLnodccz8rDhcAaOpOVCFi6Qg1mjGvHokFoVMuvEukNp2HEiS/34klBcPT4anesno0nNKm77pIe3sp7Xrdyu8rudd7tvQaEdGzduRLsOHREuEzMWf2xV3uLXFS+bp9tRyn6U8W+SmwCxJb9/K+u3zcfw8Jer1esotXFPDGhe6e90Oaz/c85OvDt3p7r8QN/GGDukpc8DSGWO32x28TJpI3708qYqaa45cFbVhshB/MCZbHW7ZAsZZeEIHLXQIPnCJgfr0iAJMx/rg69XHFAjcCTUyFCr6zvVVR+ymgmVa6s+kZGLWRuOqn1dtf+s63qp8pNmJFuhpqbhTsuxqbLINs/5y0eXHBepvmj0QKIHlOS4KNWp75QzUMhwMcepfvn8efemJ3dPDmiO0f2bFv3jkfOq6cRzs5Nms+HkpnPQml9Zel8XD+QZpDHBeipLDV+dse4w7PKSnAWubFMbTwxqpmYC9QoJUJozpBQJMyXDTV6+Da/M3IiVe08i2qrhL0Oao0eDRMft6jEcpwW2fKxcsQzdunZBuHzgnNe7Hlc9Z6GH6+1FHqes6zV7IZbvPomNh86qcHZps2Q0qx5T9N/K8+gBy/Xv5THlUKkVfeyKbK59d7++6GNodjtyc7IRHRWhzufkFyDfVgCLOlxqCLdokIECVosGS/HHr9T7dgH/xofkkFhDzmT67jmk++LA4keNM84NwOWyuVeK5sgMgM7NR2RXOskZx+8T/7r1K6DVNRf1EKv2nVGzCkvwkJF3j/e/sH5V8j0pEyvKaMxXftqCjxbuQXqODa9e3y5gJgtk+PARefMlIMj24rWt8ce2E6qJpW+zGuXORVFR8iG645IGuKpdigog36w8gOlrD6smHUnMI3s1REQZ02LL0MJfNh3DTxuOYNme067mIjmmSzPRNR3qYEjb2q4+JZKmT2flqxAi2+6TztMTmerXkNx2eu8ZrNjr/Da6QNK5SvrWyPPWqBKlOvEOalMbRmtUPQ5v39oRj1zeVP2C+N+GI2rGT9mubp+CJ/o3u/hJfdSvKWcTRxly8gvxwBersGhPDKLCG+Kju7qiR3N1uPEYuk5sy4bWdGClQleldltGCw3Q8Ov/HNW71m3AO7d1UsPP/anAZsO0H2fhYGwzfLHsgCvMtq2bgCf6N0f/VtK84uHLV4Wa4sHGEW42Hz6H//fdGhw9l4PocAvGDmmBoe1T3AJI8X9X7LE83J5fUIDFO07gl41HsPtEBiyq7sCxWSQYqfOOwBQVZkGdxCi1pSREqh84teMjUTs+Qt0mj19QYMO6NWvQsWN7hMtnyT0Aup7feaouu+2jh9sluO09lYmlu0/i6NlstT/hVg1t6ySga4OqSIwO9/CY58ueayvAliPnsO1oOmwFBaocsZFWtK7t+GESYdHK2A/n4+i3uz+2/l45r7PbC3HixAnUlBoflPNv3Mvueu7i17k/Nyr3byIvbobRXSdkpuZVyCuwY0Crml6ZU0f6Jsqsz9I/TJaWyMgtULMVB8JILoYPA8RGhuOa9r77Uk6Ki8RrN7RTSVmaYtYdPKc6BcqH7eXr2qCb/Dp2Ss+14ffNx1XgWLzzFArc+px0ql9V7efV7VI89vKXPwQJBLJd0ji5xFj0PSezXIFEDyf7TmepWhP5sEuQqB4vgSLyfLjwcBoXGRZQwzjly/JdmWzriqZqzZOfNx7FzxuOYtbGo6opaHT/Zqq2x1cyneP8l+89o5ru/j2yK3o18X8nSHmPXrq2tRqp9fWKg2ouEAm7V7b1XlCUanXpFCorFEvfKjkvQUydyvVu5+X6g2ey8M2aMOTZHUOPZUImqbaWL/MyP1Ouqv2SX8ptGsfj69G11My4Utv3+P8OY+kxK8Zd16bs/k+lNMt+tXy/qrE8lSl/eymIDK+rPkfXtE9RtX8y78yek46/H5nbocCmAadkTHDJx6uTGK1qGxsmxSAzLxrHrZ1RPzkedapGq++Fyv4dyQ8M+aH07h+7VIdSIX+7t3dLxYOXNUGdCo5okW+PzgBa5BWosv570V7HukJ7gIQj4eqH0aheDdVos4tRaLNhubP5wRrEfT6Op+di5CcrVc2yfA9Pur2z19ZTuqVrquobOPrrdeq7KyOvAB/e0Vkdl/yJfT5CjHQ2ks6or/+6DWey8tV1V7WthaTcIzgekYIFO04VWcFUvpzl16oEDl+sDSDj1OXAEB8Vbnig8NX7LYu0TZyzA79tPq4uSy2mDGmTKtILbUYrjYTFUZ+swJoD59SIpyl3d0PXhkkB9TmXz5xMVS+rCUeEWfDRnV1xecuartvzC+w4l52Ps9k2nM3OL3bepmrg5LLj+nz168wROApVcL0QMp38kwNblB86KlnOyfN24e05O9QPXalNkXWEyvu7ka9YaYKVIby/bjrmCvwyPF9qLm/rllrqQVj+fqSTuh7sXaenslx/32XVIEpYqOvc5Lz7ZfmBof8ClrLN3npc1fDp/cjk34/o0QAP9m1c6Wbc4iSg/rj2CD5csNvVF04eXyYIvL9vY9SrFmva7/T0XBtu+XCpajpvXD0O3z/cSwVHb5dbBkM88Plq9bfVtUE1/GdUN69PkskOpyH8Qa0oGcb79uztaniuW+WG65e8/kurtHHjocDX7/emw2kqhMhQQ70Z7KbO9VQNiTeCnByUZUplGVotXxIyLbx07gvEz7nUUMiifVIjJAc06Uh9NssRKErrx1NZMRFhiIkMU6dy4Dp/3v3UgviMA3hmxJWIjPRO86anL3Fpl5fAJL8oJ97WEVe0LDmhk4QnmTfms6X7sOnw+aHn0qQ5qndDNTT1Yn7dyudjzylp9szCzuPpWLV1D+wx1dSw/xMZ5S8DIZlMRthJIMnMLcDOE44OI1K7dmfPBmpiRa8M43cjTc+ztxzD+/N3F+nMLn2TpLN9mMVy/rycWuQU6nr9Otd5i8UxOCo3DTf3aY1BbesE3VwjeQWFGPXJSjW5odT6Tnu4V4W+Oy70b1xGKt396Qqk5xao5SS+uf8SJMZ67zuC4aMUZgofui1H0vH6L1uw8/ApDOvaGEM710OLWvEB1azhK0a931I9LfNe6EMP5cvz5q6pqtOx/LqV11pqRyrzmksH3Dv+vVz9GpJfQbJoVOs6CQH9OZdf6bLipvQ5Kk7KLwGqWmykmqnXcRqpZm2UPlDnr4tAQnSEOgDqoUJChixsV5HXz6iyy4gwKaveNPF/VzRVzTtycDyalqOGSkpTlF47Ifs/tGMd1dzQpo73J68rXm45qEkTj+zn4bM5OHIuF0fk/Lkc16n0LXAnNWvSFHJPn0bl/vK+WHLYkdk4ZcI6WeTRW2TU34BWtVTfng71qvpkIUQJfWsPnlUHchmZl1wlUtVcyUSQerO0nJfPd1mdO+12TS3vIJ385bX/5oFLKjyx4cV8zqXm9s7/rMAljZNUPy1vdkDlaBdykQPWf+7q4vigDmxmmtBlJKmNmHJ3d/VlJDUhi3aeUu3cshWnfqnJLzZ1QHb8cvN0nTSNST8H+TU09b7gWK1S+nvIJEdyUCmw253hwhEwJFCE0oq48gv7uwcvwas/b1XTsk/6Y5dqWqkaE6k6JOvz90ifDJnkSZpWvNXRvCKiwsNUE2BpzYB653EVRM7mqJVS5aDtzV/BZZEgKUtSyCa1Y/JZL9Q0dUCW106apuya43yh23nHqaMGRbY8mw0/zl+JI5Zk1TQpYV02mexRQsAVLWugf6tauLRZ9Qvq4yD7I7VL8retb9InpyLk454U5x5K9JDiOC/7K8FDmiplUUyjZlSWGo8fH+ut+uD5c+SLz8LH5MmT8dZbb+HYsWPo0KEDJk2ahO7du/vq6Yj8rkuDavji3h5qtM8/Z+9QVanFqalJXJWNWrkHuC/v66FG3QQLaUboW8oonFAjB3iZW0fe9+d+2KjWG9LJr0qpRZADurc6DnqTe+dxf6+CejHzA0kNQOZODVdd1R0Z+Zpae2vutuNYuOOUqj38btUhtUlToCxD399ZK5KS6Ll5RjrOS22WChoHzmLN/rOqiaI4mWxRlreQOY+kn5I+XcDpTMd5aZKT/KlfDxSd38ndWzd1QJ9mxnYgD4TmKZ+Ej2+//RZjxozBhx9+iB49emDixIkYPHgwtm/fjpo1z3dEIwpFMtHb1w9cor7IZLpu+U9NvKRpzpGFjsuu652/ku369c5TmZguEIbEUfkL78mvSZlPoV61GLW+jlwmY0lT0Y1d6qlNOjkv33sac7eeULNPHzqbo0Yqyfb8DEdHe71GRGp/9FoNaZIo3kdO+hdJE46ETNlkmvNyO4QW2l2rJMu8Rqfdwol+XVZegaoRC+T1V4IufLz99tu4//77cffdd6vLEkJ+/vlnfPLJJ3juued88ZREASeujHV9KLQ0rxWvar0oMEhov7RZDbXJcPAdxzNVCJm79TjWHjynRvTIps8AWrxWQIa76mFDgmRZ8yV5IveXEUIXO0oolHn92zE/Px+rV6/G2LFjXddZrVYMGDAAS5cuLXH/vLw8tbl3WNGr07y9Oqf+eL5Y9TOQsdwst1mYtewsd9nlbpwcjQf6NFCb1ELM33EKf2w/idX7z6FutWh0Tq2KzvWrqkXxpJN4EfZC2GR23QBiC9D3uzL74/XRLkeOHEHdunWxZMkS9OzZ03X9M888gwULFmD58uVF7j9u3DiMHz++xONMnToVsbHen3eCiIiIvC87OxvDhw8PjtEuUkMi/UPcaz5SU1MxaNAgnwy1nT17NgYOHGiqUR8sN8ttFmYtO8vNcgcCveWiIrwePqpXr46wsDAcP150rL9crl275LTLUVFRaitOXlBfvai+fOxAxnKbi1nLbeays9zmEhFg5a7Mvni9K73MKtilSxfMnTvXdZ3dbleX3ZthiIiIyJx80uwizSgjR45E165d1dweMtQ2KyvLNfqFiIiIzMsn4ePWW2/FyZMn8eKLL6pJxjp27Ihff/0VtWqVXP+AiIiIzMVnHU4fe+wxtRERERG54/SJREREZCiGDyIiIjIUwwcREREZiuGDiIiIDMXwQURERIZi+CAiIiJDMXwQERGRoRg+iIiIyFB+X9W2OE3TKr06XmVWApQlf+WxA2kxHl9juVluszBr2VluljsQ6Mdt/TgeVOEjIyNDnaampvp7V4iIiOgCjuOJiYll3seiVSSiGEhWwD1y5Aji4+NhsVi8nsok1Bw8eBAJCQkwC5ab5TYLs5ad5Wa5A4HECQkederUgdVqDa6aD9nhevXq+fQ55M0KpDfMKCy3uZi13GYuO8ttLgkBWO7yajx07HBKREREhmL4ICIiIkOZKnxERUXhpZdeUqdmwnKz3GZh1rKz3Cx3sAm4DqdEREQU2kxV80FERET+x/BBREREhmL4ICIiIkMxfBAREZGhTBM+Jk+ejIYNGyI6Oho9evTAihUrEExee+01dOvWTc38WrNmTQwbNgzbt28vcp/c3Fw8+uijSE5ORpUqVXDjjTfi+PHjRe5z4MABXH311YiNjVWP8/TTT6OgoKDIfebPn4/OnTurntRNmzbFlClTEAhef/11NevtE088YYoyHz58GHfccYcqW0xMDNq1a4dVq1a5bpe+4i+++CJSUlLU7QMGDMDOnTuLPMaZM2cwYsQINRFR1apVce+99yIzM7PIfTZs2IBLL71U/W3IrIlvvvkm/KWwsBAvvPACGjVqpMrUpEkTvPLKK0XWigiFci9cuBDXXnutmglSPtMzZswocruRZfzvf/+Lli1bqvvIZ2zWrFnwV9llzZJnn31W7UdcXJy6z1133aVmvQ72spf3nrt76KGH1H0mTpwY9OUulWYC33zzjRYZGal98skn2ubNm7X7779fq1q1qnb8+HEtWAwePFj79NNPtU2bNmnr1q3TrrrqKq1+/fpaZmam6z4PPfSQlpqaqs2dO1dbtWqVdskll2i9evVy3V5QUKC1bdtWGzBggLZ27Vpt1qxZWvXq1bWxY8e67rNnzx4tNjZWGzNmjLZlyxZt0qRJWlhYmPbrr79q/rRixQqtYcOGWvv27bXHH3885Mt85swZrUGDBtqoUaO05cuXq3387bfftF27drnu8/rrr2uJiYnajBkztPXr12vXXXed1qhRIy0nJ8d1nyuvvFLr0KGDtmzZMm3RokVa06ZNtdtvv911e1pamlarVi1txIgR6rP19ddfazExMdq//vUvzR9effVVLTk5Wfvpp5+0vXv3av/973+1KlWqaO+8805IlVs+h3/961+1adOmSarSpk+fXuR2o8r4559/qs/6m2++qT77zz//vBYREaFt3LjRL2U/d+6c+lv99ttvtW3btmlLly7VunfvrnXp0qXIYwRj2ct7z3Vyu5StTp062j//+U8t2MtdGlOED/nwPvroo67LhYWF6o197bXXtGB14sQJ9QFesGCB649WPkDyZa3bunWruo/8AesffqvVqh07dsx1nw8++EBLSEjQ8vLy1OVnnnlGa9OmTZHnuvXWW1X48ZeMjAytWbNm2uzZs7XLLrvMFT5CuczPPvus1qdPn1Jvt9vtWu3atbW33nrLdZ28HlFRUeoLR8gXi7wWK1eudN3nl19+0SwWi3b48GF1+f3339eqVavmei30527RooXmD1dffbV2zz33FLnuhhtuUF+moVru4gciI8t4yy23qNfcXY8ePbQHH3xQM0JZB2H3Hx5yv/3794dM2VFKuQ8dOqTVrVtXBQf58eEePkKh3O5CvtklPz8fq1evVtWW7uvHyOWlS5ciWKWlpanTpKQkdSpllCpL93JKtVr9+vVd5ZRTqWKrVauW6z6DBw9WixRt3rzZdR/3x9Dv48/XSppVpNmk+H6FcplnzpyJrl274uabb1ZNRZ06dcLHH3/sun3v3r04duxYkf2WNRWkSdG97FI1K4+jk/vL53/58uWu+/Tt2xeRkZFFyi5NemfPnoXRevXqhblz52LHjh3q8vr167F48WIMGTIkpMvtzsgyBuJn39N3nTRBSHlDuex2ux133nmnahZu06ZNidtDrdwhHz5OnTql2pHdDz5CLssfeDCSD6n0e+jduzfatm2rrpOyyAdO/wP1VE459fQ66LeVdR85WOfk5MBo33zzDdasWaP6vBQXqmUWe/bswQcffIBmzZrht99+w8MPP4zRo0fjs88+K7LvZX2u5VSCi7vw8HAVWCvz+hjpueeew2233aZCZEREhApd8lmXdu5QLrc7I8tY2n38/Rq49+mSPiC33367awG1UC37G2+8ocohf+eehFq5A25VW6pYTcCmTZvUL8JQJstFP/7445g9e7bqGGUmEjDlF86ECRPUZTkIy3v+4YcfYuTIkQhV3333Hb766itMnTpV/fpbt26dCh/SSS+Uy00lSa3mLbfcojrfShAPZatXr8Y777yjfmhJLY8ZhHzNR/Xq1REWFlZiBIRcrl27NoLNY489hp9++gnz5s1DvXr1XNdLWaSJ6dy5c6WWU049vQ76bWXdR351SK97o/8gT5w4oUahSMKXbcGCBXj33XfVeUnroVZmnYxyaN26dZHrWrVqpUbuuO97WZ9rOZXXz52M8pEe85V5fYwkVc567Yc0l0k19JNPPumq+QrVcrszsoyl3cffr4EePPbv369+fLgvGx+KZV+0aJEqkzQZ6991UvannnpKjdIMxXKHfPiQavkuXbqodmT3X5VyuWfPnggWkv4leEyfPh1//PGHGoroTsoo1dTu5ZR2PjlY6eWU040bNxb5AOt/2PqBTu7j/hj6ffzxWvXv31/tr/z61TepDZAqeP18qJVZJ01qxYdSSz+IBg0aqPPy/suXhft+SzORtP26l12CmYQ4nXx25PMv/Qf0+8gQQPmydy97ixYtUK1aNRgtOztbtWG7kx8Pss+hXG53RpYxED/7evCQocVz5sxRQ83dhWLZ77zzTjVE1v27Tmr7JIxLs2tIllszyVBb6Sk+ZcoU1WP4gQceUENt3UdABLqHH35YDb2bP3++dvToUdeWnZ1dZNipDL/9448/1LDTnj17qq34sNNBgwap4boylLRGjRoeh50+/fTTauTI5MmT/T7s1J37aJdQLrP08A8PD1dDT3fu3Kl99dVXah+//PLLIsMx5XP8448/ahs2bNCGDh3qcThmp06d1HDdxYsXq1FD7kPzZBSFDM278847VQ97+VuR5/HXUNuRI0eq3v76UFsZdihDo2VEUiiVW0ZwydBv2eRr+O2331bn9REdRpVRhl3K5+zvf/+7+uy/9NJLPh92WVbZ8/Pz1bDievXqqb9X9+869xEcwVj28t7z4oqPdgnWcpfGFOFDyNwNcpCS+T5k6K2Mkw4m8mH1tMncHzr5YnrkkUfUUCv5wF1//fXqj9bdvn37tCFDhqix3/Kl/tRTT2k2m63IfebNm6d17NhRvVaNGzcu8hyBFj5Cucz/+9//VHCS4NyyZUvto48+KnK7DMl84YUX1JeN3Kd///7a9u3bi9zn9OnT6stJ5sqQ4cV33323+hJ0J/NIyLBeeQw58MuBz1/S09PV+yt/q9HR0eq9kLkR3A88oVBu+bx5+nuW8GV0Gb/77jutefPm6rMvQ85//vlnv5VdAmdp33Xy74K57OW95xUJH8FY7tJY5H/G1rUQERGRmYV8nw8iIiIKLAwfREREZCiGDyIiIjIUwwcREREZiuGDiIiIDMXwQURERIZi+CAiIiJDMXwQERGRoRg+iIiIyFAMH0TkdaNGjcKwYcP8vRtEFKAYPoiIiMhQDB9EdMG+//57tGvXDjExMWrp8wEDBqhlwD/77DP8+OOPsFgsaps/f766/8GDB9Vy6VWrVkVSUhKGDh2Kffv2lagxGT9+PGrUqIGEhAQ89NBDyM/P92Mpicjbwr3+iERkCkePHsXtt9+ON998E9dffz0yMjKwaNEi3HXXXThw4ADS09Px6aefqvtK0LDZbBg8eDB69uyp7hceHo6//e1vuPLKK7FhwwZERkaq+86dOxfR0dEqsEgwufvuu1WwefXVV/1cYiLyFoYPIrrg8FFQUIAbbrgBDRo0UNdJLYiQmpC8vDzUrl3bdf8vv/wSdrsd//73v1VtiJBwIrUgEjQGDRqkrpMQ8sknnyA2NhZt2rTByy+/rGpTXnnlFVitrKwlCgX8SyaiC9KhQwf0799fBY6bb74ZH3/8Mc6ePVvq/devX49du3YhPj4eVapUUZvUiOTm5mL37t1FHleCh05qSjIzM1WTDRGFBtZ8ENEFCQsLw+zZs7FkyRL8/vvvmDRpEv76179i+fLlHu8vAaJLly746quvStwm/TuIyDwYPojogknzSe/evdX24osvquaX6dOnq6aTwsLCIvft3Lkzvv32W9SsWVN1JC2rhiQnJ0c13Yhly5apWpLU1FSfl4eIjMFmFyK6IFLDMWHCBKxatUp1MJ02bRpOnjyJVq1aoWHDhqoT6fbt23Hq1CnV2XTEiBGoXr26GuEiHU737t2r+nqMHj0ahw4dcj2ujGy59957sWXLFsyaNQsvvfQSHnvsMfb3IAohrPkgogsitRcLFy7ExIkT1cgWqfX4xz/+gSFDhqBr164qWMipNLfMmzcP/fr1U/d/9tlnVSdVGR1Tt25d1W/EvSZELjdr1gx9+/ZVnVZlRM24ceP8WlYi8i6Lpmmalx+TiOiCyDwf586dw4wZM/y9K0TkQ6zHJCIiIkMxfBAREZGh2OxCREREhmLNBxERERmK4YOIiIgMxfBBREREhmL4ICIiIkMxfBAREZGhGD6IiIjIUAwfREREZCiGDyIiIoKR/j/0mcMWAchn1AAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 24
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:03.914476Z",
     "start_time": "2025-01-17T14:35:03.793510Z"
    }
   },
   "source": [
    "model.eval()\n",
    "loss = evaluating(model, val_loader, loss_fct)\n",
    "print(f\"loss:     {loss:.4f}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:     0.4226\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拿到中间输出"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:03.920922Z",
     "start_time": "2025-01-17T14:35:03.915478Z"
    }
   },
   "source": [
    "logits, deep_output = model(train_ds[:][0].to(device), return_deep_output=True)"
   ],
   "outputs": [],
   "execution_count": 26
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:03.925924Z",
     "start_time": "2025-01-17T14:35:03.921926Z"
    }
   },
   "source": [
    "deep_output.shape"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([11610, 30])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 27
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T14:35:04.028994Z",
     "start_time": "2025-01-17T14:35:03.926923Z"
    }
   },
   "source": [
    "# 从这看到deep部分抽取到的特征分布，有些特征没那么重要，或许可以消减一些神经元\n",
    "plt.imshow(deep_output.cpu().detach().numpy().mean(axis=0).reshape(1, -1))\n",
    "plt.yticks([])\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAA8CAYAAAD7aLkGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAACV1JREFUeJzt3XtIVH0aB/BnLjqWZmUXL3npYhe2i8tWVkQWa9hliW5/dNtdizAqiy50ocAsCIT6J4qW2H962aWkhCwK3oUoMwKtpYgISrK3XRU1KdZ7pjPzW54fO+K8r23qc+o47+/7gWGaPOf4+Mwz5zxzzu+c41BKKQIAAABjOe0OAAAAAOyFZgAAAMBwaAYAAAAMh2YAAADAcGgGAAAADIdmAAAAwHBoBgAAAAyHZgAAAMBw7r5M5Pf7qba2loYNG0YOh+PbRwUAAABifF3BlpYWSkhIIKfTKWsGuBFISkqSRwUAAADfXXV1NSUmJsqaAd4jwHL+8QcKjwwbUCAVf/pyEH3l/Ve1eBmu1Ami+VunxYhjCGv1iea//de/i2P4/YntovmdXfKrWMfv+Ek0f8efXeIY/v1HWT3EL6kRx1BTPk40/28y3opjqLoySTR/zN+eiGNwJyaIl+H/T6NofmdUlDyGpibR/I7ISHEMHb9NEc3ved8mjsHZ0i6aX7V1iGOgzs+i2dN+lNUT++ex34nmD69rEc3v9X2m0p/+0r0dFzUDgUMD3Ah4ogbWDLidngHNFxzIwH53Ty6XLA53WIQ4Brdb1gxED5MP9XCFy/4OF8mbgbDIcNH8Xqe8GXB5ZHlwR8rr2hURYWseragHtwWfTSvWEX6HLBdOZ7jtMTgsiMHtFr6fLq84BqdTtp5TTr84BnLI1lOeAW7vrH0vOskKXzvEjwGEAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGQzMAAABgODQDAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGQzMAAABgOHdf74fMOtu6BvyLvH7Z3aP0MtTAf3+A8sni8HbJ76Tl8Mpu4NHcIr+Bh69T9ncoC+5a2NUmuwGH1y+/UZHvsywP3jZ5Xfs6OmzNoxX1YMVnkyxYR/iVLBdOf6ftMTj88pvjeL2y99MlXE8yp/D9VBa8FyR8Lz63dtn+Xjil26z/zR/Yjn+JQ31tCr7Fak0NJSUliQICAAAAe1RXV1NiYqKsGfD7/VRbW6vvh9zbbRCbm5t1s8C/LDo6Wh61wZBL6yCX1kAerYNcWge57BvexLe0tFBCQgI5nU7ZYQJewP/rKAL4DcGbYg3k0jrIpTWQR+sgl9ZBLr9u+PDhX50GAwgBAAAMh2YAAADAcJY0Ax6Ph/Lz8/UzyCCX1kEurYE8Wge5tA5yaa0+DSAEAACAXy8cJgAAADAcmgEAAADDoRkAAAAwHJoBAAAAw1nSDFy8eJHGjx9PERERNG/ePHry5IkVizXKyZMn9dUdez6mTZtmd1iD3sOHD2nVqlX66lqcs5s3bwb9nMfHnjhxguLj42nIkCG0dOlSevPmjW3xhnIut27d+osaXb58uW3xDlYFBQU0d+5cfcXWsWPH0po1a6iioiJomo6ODsrNzaVRo0ZRVFQUrV+/nt6/f29bzKGcyyVLlvyiLnfu3GlbzMY2A9euXaODBw/qUzyePXtGaWlptGzZMmpoaLAmQoNMnz6d6urquh+PHj2yO6RBr62tTdccN6S9OXPmDJ0/f54uXbpEjx8/psjISF2fvDKG/uWS8ca/Z40WFhZ+1xhDQWlpqd7Ql5eX0927d6mrq4uysrJ0fgMOHDhAt2/fpqKiIj09X+593bp1tsYdqrlkOTk5QXXJn3voJyWUnp6ucnNzu1/7fD6VkJCgCgoKpIs2Sn5+vkpLS7M7jJDG5VxcXNz92u/3q7i4OHX27Nnu/2tsbFQej0cVFhbaFGVo5pJlZ2er1atX2xZTqGpoaND5LC0t7a7BsLAwVVRU1D3Nq1ev9DRlZWU2Rhp6uWSLFy9W+/btszWuXwPRnoHOzk56+vSp3vXa8z4G/LqsrEyyaCPx7mveRTtx4kTasmULVVVV2R1SSHv37h3V19cH1Sdfo5sPZaE+B+bBgwd6d+3UqVNp165d9PHjR7tDGvSampr0c0xMjH7mdSZ/w+1Zl3xIMDk5GXXZz1wGXLlyhUaPHk0zZsygY8eOUXt7u00Rhq4+3ajoSz58+EA+n49iY2OD/p9fv379WhqbUXgD9cMPP+iVLO/mOnXqFC1atIhevnypj5dB/3EjwHqrz8DPoO/4EAHvyp4wYQK9ffuWjh8/TitWrNAbMJfLZXd4gxLf8XX//v20cOFCvaFiXHvh4eE0YsSIoGlRl/3PJdu8eTOlpKToL1IvXrygo0eP6nEFN27csDVeo5oBsA6vVANmzZqlmwMu8OvXr9P27dttjQ2Abdy4sfvfM2fO1HU6adIkvbcgMzPT1tgGKz7ezQ09xv98u1zu2LEjqC55sDDXIzesXJ/QN6LDBLxbhr8R/HwULL+Oi4uTLNp4/K1hypQpVFlZaXcoIStQg6jPb4MPZ/E6ADXauz179tCdO3eopKQk6BbwXHt8iLWxsTFoetRl/3PZG/4ixVCX37EZ4F1ds2fPpnv37gXtyuHXCxYskCzaeK2trbqz5S4XBoZ3Z/PKtWd9Njc367MKUJ9yNTU1eswAajQYj7/kjVdxcTHdv39f12FPvM4MCwsLqkverc1jhFCX/ctlb54/f66fUZff+TABn1aYnZ1Nc+bMofT0dDp37pw+7WPbtm3SRRvl0KFD+hxvPjTApxnxqZq812XTpk12hzbom6ae3wB40CCvDHiAEQ/I4mOMp0+fpsmTJ+sVSV5enj62yOcrQ99zyQ8ex8Lnw3ODxY3qkSNHKDU1VZ+qCcG7s69evUq3bt3S430C4wB48Cpf64Kf+dAfrzs5r9HR0bR3717dCMyfP9/u8EMql1yH/POVK1fqazbwmAE+bTMjI0MfxoJ+sOKUhAsXLqjk5GQVHh6uTzUsLy+3YrFG2bBhg4qPj9c5HDdunH5dWVlpd1iDXklJiT7V6OcPPg0ucHphXl6eio2N1acUZmZmqoqKCrvDDrlctre3q6ysLDVmzBh9WlxKSorKyclR9fX1doc96PSWQ35cvny5e5pPnz6p3bt3q5EjR6qhQ4eqtWvXqrq6OlvjDsVcVlVVqYyMDBUTE6M/36mpqerw4cOqqanJ7tBDDm5hDAAAYDjcmwAAAMBwaAYAAAAMh2YAAADAcGgGAAAADIdmAAAAwHBoBgAAAAyHZgAAAMBwaAYAAAAMh2YAAADAcGgGAAAADIdmAAAAwHBoBgAAAMhs/wU1bTQj8Qn9OAAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 28
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
